package com.g2room.driving.grap.article.kc;

import java.io.IOException;
import java.net.MalformedURLException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.g2room.driving.grap.GenericGrap;
import com.g2room.driving.grap.article.HandleImages;
import com.g2room.driving.grap.article.to.ArticlesTo;
import com.g2room.driving.util.DbUtil;
import com.g2room.driving.util.HtmlUtil;
import com.g2room.driving.util.IpUtil;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.mysql.jdbc.Statement;

public class GrabCcwDataByProxy {
	static Connection connection = null;
	static PreparedStatement  ps = null;
	static String baseUrl = "http://www.checar.cn";
	static Map<String, String> errorUrls = new HashMap<String, String>();
//	static List<Integer> ids = new ArrayList<Integer>();
	private static final String soruce = "车车网";
	private int count = 0;
	
	public static void main(String[] args) throws Exception{
		GrabCcwDataByProxy gkd = new GrabCcwDataByProxy();
		gkd.getNav();
		
//		// 抓取图片
//		if(null != ids && ids.size() > 0) {
//			HandleImages.getImgs(ids, null);
//		}
    }
	
	
	/**
	 * 获取头部导航菜单
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public void getNav() throws FailingHttpStatusCodeException, MalformedURLException, IOException {
		Map<String, String> pages = new HashMap<String, String>();
//		pages.put("http://www.checar.cn/usecar/17/37/page16.shtml", "车险知识");
//		pages.put("http://www.checar.cn/usecar/13/page46.shtml", "驾驶技巧");
//		pages.put("http://www.checar.cn/usecar/12/page13.shtml", "省油技巧");
//		pages.put("http://www.checar.cn/usecar/17/38/page27.shtml", "保险理赔");
//		pages.put("http://www.checar.cn/live/24/42/page03.shtml", "汽车故事");
//		pages.put("http://www.checar.cn/live/24/26/page12.shtml", "囧闻趣事");
//		pages.put("http://www.checar.cn/usecar/20/page05.shtml", "汽车百科");
//		pages.put("http://www.checar.cn/usecar/14/page148.shtml", "维修保养");
//		pages.put("http://www.checar.cn/news/5/page24.shtml", "政策法规");
//
//        for(String key : pages.keySet()) {
//	        String pageTotal = key.substring(key.indexOf("page") + 4, key.lastIndexOf("."));
//	        for(int i = 0; i<Integer.valueOf(pageTotal); i++) {
//	        	boolean rs = getPage(key.replaceAll("page" + pageTotal, "page" + i), pages.get(key));
//	        	if(!rs) break;
//	        }
//        }

		pages.put("http://www.checar.cn/usecar/17/37/page1.shtml", "车险知识");
		pages.put("http://www.checar.cn/usecar/13/page1.shtml", "驾驶技巧");
		pages.put("http://www.checar.cn/usecar/12/page1.shtml", "省油技巧");
		pages.put("http://www.checar.cn/usecar/17/38/page1.shtml", "保险理赔");
		pages.put("http://www.checar.cn/live/24/42/page1.shtml", "汽车故事");
		pages.put("http://www.checar.cn/live/24/26/page1.shtml", "囧闻趣事");
		pages.put("http://www.checar.cn/usecar/20/page1.shtml", "汽车百科");
		pages.put("http://www.checar.cn/usecar/14/page1.shtml", "维修保养");
		pages.put("http://www.checar.cn/news/5/page1.shtml", "政策法规");

        for(String key : pages.keySet()) {
	        getPage(key, pages.get(key));
        }
        System.out.println("本次抓取数据：" + count);
		
//		// 抓取图片
//		if(null != ids && ids.size() > 0) {
//			HandleImages.getImgs(ids, null);
//		}
		// 最后输出失败的链接到数据库
		insertError2Db();
	}
	
	/**
	 * 获取分页数据
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public boolean getPage(String url, String typeName) throws FailingHttpStatusCodeException, MalformedURLException, IOException {
		System.out.println("start get page data!" + typeName);
		Document doc = GenericGrap.getDoc(url);
		Elements els = doc.getElementById("con_nineteen_1").getElementsByTag("li");
		List<ArticlesTo> artis = new ArrayList<ArticlesTo>();
		for(Element el : els) {
			try {
				Element typeEl = el.getElementsByTag("a").get(0);
				Element hrefEl = el.getElementsByTag("a").get(1);
				String title = hrefEl.text();
//				if(GenericGrap.titles.contains(title)) {
//					System.out.println("存在，跳过：" + title);
//					if(artis.size() == 0) {
//						// 后续抓取的时候，就存在则直接返回
//						return false;
//					} else {
//						break;
//					}
//				}
				
				ArticlesTo arti = new ArticlesTo();
				arti.setTitle(title);
				arti.setSource(soruce);
				arti.setSourceUrl(baseUrl + hrefEl.attr("href"));
				arti.setTypeName(typeEl.text().replace("[", "").replace("]", ""));
				arti.setState(7);
				artis.add(arti);
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
		
        for(ArticlesTo arti : artis) {
        	try {
				Thread.sleep(IpUtil.genRandomNum() * 500);
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
        	getDetail(arti);
        }
        return true;
	}
	
	public void getDetail(ArticlesTo arti) throws FailingHttpStatusCodeException, MalformedURLException, IOException {
		System.out.println("start get detail data!  " + arti.getSourceUrl());
		try {
			Document doc = GenericGrap.getDoc(arti.getSourceUrl());
			String content = doc.getElementsByAttributeValue("class", "news_con").html();
			content = handleSpace(HtmlUtil.delAllTag(content));
			arti.setContent("<div class=\"article\">" + content + "</div>");
			// 获取发布时间
			Elements time = doc.getElementsByAttributeValue("class", " cor666");
			String publishTime = time.get(0).text();
			
			arti.setPublishTime(publishTime);
			insert2Db(arti);
//			GenericGrap.titles.add(arti.getTitle());
			count++;
        } catch (Exception e) {
        	errorUrls.put(arti.getSourceUrl(), arti.getTypeName());
        	e.printStackTrace();
        	return;
        }
	}
	int sqlCount = 0;
	public void insert2Db(ArticlesTo article) {
		try {
			if(sqlCount == 0) {
				ps = DbUtil.getConn().prepareStatement("insert into d_article_grap(title,content,publish_time,type_name,source,source_url,state) values(?,?,?,?,?,?,?)", Statement.RETURN_GENERATED_KEYS);
			}
			ps.setString(1, article.getTitle());
			ps.setString(2, article.getContent());
			ps.setString(3, article.getPublishTime());
			ps.setString(4, article.getTypeName());
			ps.setString(5, article.getSource());
			ps.setString(6, article.getSourceUrl());
			ps.setInt(7, article.getState());
			ps.execute();
			
//			ResultSet rs = ps.getGeneratedKeys(); //获取结果   
//			if (rs.next()) {
//				ids.add(rs.getInt(1));
//			}
		} catch(Exception e) {
			System.out.println(article.getTitle() + "=url==" + article.getSourceUrl());
			e.printStackTrace();
		}
	}
	
	public static void insertError2Db() {
		try {
			
			PreparedStatement  ps = DbUtil.getConn().prepareStatement("insert into d_article_grap_err(title,source,source_url,type_name) values(?,?,?,?)");
			for(String url : errorUrls.keySet()) {
				ps.setString(1, errorUrls.get(url));
				ps.setString(2, soruce);
				ps.setString(3, url);
				ps.setString(4, "导航和列表错误");
				ps.addBatch();
			}
			ps.executeBatch();
			if(null != ps) ps.close();
			DbUtil.closeConn();
		} catch(Exception e) {
			e.printStackTrace();
		}
	}
	
	public static String handleSpace(String str) {
		return str.replaceAll("&nbsp;", " ");
	}
}
